<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Latency Analyzer</title>
    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.3.6/dist/css/bootstrap.min.css" rel="stylesheet" integrity="sha384-4Q6Gf2aSP4eDXB8Miphtr37CMZZQ5oXLH2yaXMJ2w8e2ZtHTl7GptT4jmndRuHDT" crossorigin="anonymous">
    <script src="https://unpkg.com/@popperjs/core@2"></script>
    <script src="https://unpkg.com/tippy.js@6"></script>
    <link rel="stylesheet" href="https://unpkg.com/tippy.js@6/dist/tippy.css">
    <style>
        .matrix-legend {
            margin-left: 20px;
            margin-right: 20px;
        }
        .cell {
            stroke: #ccc;
        }
        .cell.unlight {
            opacity: 0.2;
        }
        .axis text {
            font-size: 12px;
        }

        /* Custom Tippy theme */
        .tippy-box[data-theme~='latency-data'] {
            background-color: rgba(0, 0, 0, 0.8);
            color: white;
            font-size: 12px;
            border-radius: 4px;
            padding: 5px;
            box-shadow: 0 0 10px rgba(0, 0, 0, 0.5);
        }

        #hourglass
        {
            display: none;
            font-size: 110%;
            color: #888;
            animation: hourglass-animation 2s linear infinite;
        }

        @keyframes hourglass-animation {
            0% { transform: none; }
            25% { transform: rotate(180deg); }
            50% { transform: rotate(180deg); }
            75% { transform: rotate(360deg); }
            100% { transform: rotate(360deg); }
        }
    </style>
</head>
<body>
    <script src="https://d3js.org/d3.v7.min.js"></script>
    <script src="https://cdn.jsdelivr.net/npm/bootstrap@5.3.6/dist/js/bootstrap.bundle.min.js" integrity="sha384-j1CDi7MgGQ12Z7Qab0qlWQ/Qqz24Gc6BM0thvEMVjHnfYGF0rmFCozFSxQBxwHKO" crossorigin="anonymous"></script>
    <script src="https://cdn.jsdelivr.net/npm/sorttable@1.0.2/sorttable.min.js"></script>
    <script language="javascript" type="text/javascript" src="js/query-clickhouse.js"></script>
    <script language="javascript" type="text/javascript" src="js/data-visualization.js"></script>
    <script language="javascript" type="text/javascript" src="js/data-processing.js"></script>

    <nav class="navbar navbar-expand-lg navbar-light bg-light">
        <div class="container-fluid">
            <a class="navbar-brand" href="#">ClickHouse query-latency-analyzer</a>
            <button class="navbar-toggler" type="button"
                    data-bs-toggle="collapse"
                    data-bs-target="#bs-example-navbar-collapse-1"
                    aria-controls="bs-example-navbar-collapse-1"
                    aria-expanded="false"
                    aria-label="Toggle navigation">
                <span class="navbar-toggler-icon"></span>
            </button>
            <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
                <form class="d-flex">
                    <button type="button" class="btn btn-primary"
                            id="toolbar-load"
                            data-bs-toggle="modal"
                            data-bs-target="#loadModal">
                        Load
                    </button>
                </form>
                <p class="navbar-text ms-auto" id="status-text"></p>
            </div>
        </div>
    </nav>

    <div class="modal fade" id="loadModal" tabindex="-1" aria-labelledby="loadModalLabel" aria-hidden="true">
        <div class="modal-dialog modal-lg">
            <div class="modal-content">
                <div class="modal-header">
                    <h5 class="modal-title" id="loadModalLabel">Load Data</h5>
                    <button type="button" class="btn-close" data-bs-dismiss="modal" aria-label="Close"></button>
                </div>
                <div class="modal-body">
                    <ul class="nav nav-tabs" id="loadModalTabs" role="tablist">
                        <li class="nav-item" role="presentation">
                            <button class="nav-link active" id="upload-tab" data-bs-toggle="tab"
                                data-bs-target="#tabFileUpload" type="button" role="tab">Upload JSON</button>
                        </li>
                        <li class="nav-item" role="presentation">
                            <button class="nav-link" id="clickhouse-tab" data-bs-toggle="tab"
                                data-bs-target="#tabClickhouse" type="button" role="tab">From ClickHouse</button>
                        </li>
                    </ul>

                    <div class="tab-content mt-3">
                        <div class="tab-pane fade show active" id="tabFileUpload" role="tabpanel">
                            <input type="file" id="loadFiles" class="form-control" />
                        </div>
                        <div class="tab-pane fade" id="tabClickhouse" role="tabpanel">
                            <form id="params">
                                <div id="connection-params" class="mb-3">
                                    <input spellcheck="false" id="url" type="text" value="http://localhost:8123"
                                        placeholder="URL" class="form-control mb-2" />
                                    <input spellcheck="false" id="user" type="text" value="" placeholder="user"
                                        class="form-control mb-2" />
                                    <input spellcheck="false" id="password" type="password" placeholder="password" value=""
                                        class="form-control mb-2" />
                                </div>
                                <textarea spellcheck="false" id="query" rows="12" class="form-control"
                                    placeholder="SQL Query">WITH 1 AS last_hours_to_query
SELECT
    normalized_query_hash as group,
    query_duration_ms * 1000 as _duration,
    intDiv(ProfileEvents['LoggerElapsedNanoseconds'], 1000) as LoggerElapsedNanoseconds,
    ProfileEvents['GlobalThreadPoolLockWaitMicroseconds'] as GlobalThreadPoolLockWait,
    ProfileEvents['LocalThreadPoolLockWaitMicroseconds'] as LocalThreadPoolLockWait,
    ProfileEvents['SchedulerIOReadWaitMicroseconds'] as SchedulerIOReadWait,
    ProfileEvents['SchedulerIOWriteWaitMicroseconds'] as SchedulerIOWriteWait,
    ProfileEvents['ZooKeeperWaitMicroseconds'] as ZooKeeperWait,
    ProfileEvents['ContextLockWaitMicroseconds'] as ContextLockWait,
    ProfileEvents['PartsLockWaitMicroseconds'] as PartsLockWait,
    ProfileEvents['AsynchronousReadWaitMicroseconds'] as AsynchronousReadWait,
    ProfileEvents['SynchronousReadWaitMicroseconds'] as SynchronousReadWait,
    ProfileEvents['AsynchronousRemoteReadWaitMicroseconds'] as AsynchronousRemoteReadWait,
    ProfileEvents['SynchronousRemoteReadWaitMicroseconds'] as SynchronousRemoteReadWait,
    ProfileEvents['AsyncLoaderWaitMicroseconds'] as AsyncLoaderWait,
    ProfileEvents['ConcurrencyControlWaitMicroseconds'] as ConcurrencyControlWait,
    ProfileEvents['FileSyncElapsedMicroseconds'] as FileSyncElapsed,
    ProfileEvents['DirectorySyncElapsedMicroseconds'] as DirectorySyncElapsed,
    ProfileEvents['CompressedReadBufferChecksumDoesntMatchMicroseconds'] as CompressedReadBufferChecksumDoesntMatch,
    ProfileEvents['OpenedFileCacheMicroseconds'] as OpenedFileCache,
    ProfileEvents['DiskReadElapsedMicroseconds'] as DiskReadElapsed,
    ProfileEvents['DiskWriteElapsedMicroseconds'] as DiskWriteElapsed,
    ProfileEvents['NetworkReceiveElapsedMicroseconds'] as NetworkReceiveElapsed,
    ProfileEvents['NetworkSendElapsedMicroseconds'] as NetworkSendElapsed,
    ProfileEvents['GlobalThreadPoolThreadCreationMicroseconds'] as GlobalThreadPoolThreadCreation,
    ProfileEvents['LocalThreadPoolThreadCreationMicroseconds'] as LocalThreadPoolThreadCreation,
    ProfileEvents['DiskS3GetRequestThrottlerSleepMicroseconds'] as DiskS3GetRequestThrottlerSleep,
    ProfileEvents['DiskS3PutRequestThrottlerSleepMicroseconds'] as DiskS3PutRequestThrottlerSleep,
    ProfileEvents['S3GetRequestThrottlerSleepMicroseconds'] as S3GetRequestThrottlerSleep,
    ProfileEvents['S3PutRequestThrottlerSleepMicroseconds'] as S3PutRequestThrottlerSleep,
    ProfileEvents['RemoteReadThrottlerSleepMicroseconds'] as RemoteReadThrottlerSleep,
    ProfileEvents['RemoteWriteThrottlerSleepMicroseconds'] as RemoteWriteThrottlerSleep,
    ProfileEvents['LocalReadThrottlerSleepMicroseconds'] as LocalReadThrottlerSleep,
    ProfileEvents['LocalWriteThrottlerSleepMicroseconds'] as LocalWriteThrottlerSleep,
    ProfileEvents['CompileExpressionsMicroseconds'] as CompileExpressions,
    ProfileEvents['FilteringMarksWithPrimaryKeyMicroseconds'] as FilteringMarksWithPrimaryKey,
    ProfileEvents['FilteringMarksWithSecondaryKeysMicroseconds'] as FilteringMarksWithSecondaryKeys,
    ProfileEvents['WaitMarksLoadMicroseconds'] as WaitMarksLoad,
    ProfileEvents['MemoryOvercommitWaitTimeMicroseconds'] as MemoryOvercommitWaitTime,
    ProfileEvents['MemoryAllocatorPurgeTimeMicroseconds'] as MemoryAllocatorPurgeTime,
    ProfileEvents['ParallelReplicasHandleRequestMicroseconds'] as ParallelReplicasHandleRequest,
    ProfileEvents['ParallelReplicasHandleAnnouncementMicroseconds'] as ParallelReplicasHandleAnnouncement,
    ProfileEvents['ParallelReplicasAnnouncementMicroseconds'] as ParallelReplicasAnnouncement,
    ProfileEvents['ParallelReplicasReadRequestMicroseconds'] as ParallelReplicasReadRequest,
    ProfileEvents['ParallelReplicasStealingByHashMicroseconds'] as ParallelReplicasStealingByHash,
    ProfileEvents['ParallelReplicasProcessingPartsMicroseconds'] as ParallelReplicasProcessingParts,
    ProfileEvents['ParallelReplicasStealingLeftoversMicroseconds'] as ParallelReplicasStealingLeftovers,
    ProfileEvents['ParallelReplicasCollectingOwnedSegmentsMicroseconds'] as ParallelReplicasCollectingOwnedSegments,
    ProfileEvents['S3ReadMicroseconds'] as S3Read,
    ProfileEvents['S3WriteMicroseconds'] as S3Write,
    ProfileEvents['DiskS3ReadMicroseconds'] as DiskS3Read,
    ProfileEvents['DiskS3WriteMicroseconds'] as DiskS3Write,
    ProfileEvents['ReadBufferFromS3Microseconds'] as ReadBufferFromS3,
    ProfileEvents['ReadBufferFromS3InitMicroseconds'] as ReadBufferFromS3Init,
    ProfileEvents['WriteBufferFromS3Microseconds'] as WriteBufferFromS3,
    ProfileEvents['WriteBufferFromS3WaitInflightLimitMicroseconds'] as WriteBufferFromS3WaitInflightLimit,
    ProfileEvents['ReadBufferFromAzureMicroseconds'] as ReadBufferFromAzure,
    ProfileEvents['ReadBufferFromAzureInitMicroseconds'] as ReadBufferFromAzureInit,
    ProfileEvents['CachedReadBufferReadFromSourceMicroseconds'] as CachedReadBufferReadFromSource,
    ProfileEvents['CachedReadBufferReadFromCacheMicroseconds'] as CachedReadBufferReadFromCache,
    ProfileEvents['CachedReadBufferCacheWriteMicroseconds'] as CachedReadBufferCacheWrite,
    ProfileEvents['CachedReadBufferCreateBufferMicroseconds'] as CachedReadBufferCreateBuffer,
    ProfileEvents['CachedWriteBufferCacheWriteMicroseconds'] as CachedWriteBufferCacheWrite,
    ProfileEvents['FilesystemCacheLoadMetadataMicroseconds'] as FilesystemCacheLoadMetadata,
    ProfileEvents['FilesystemCacheLockKeyMicroseconds'] as FilesystemCacheLockKey,
    ProfileEvents['FilesystemCacheLockMetadataMicroseconds'] as FilesystemCacheLockMetadata,
    ProfileEvents['FilesystemCacheLockCacheMicroseconds'] as FilesystemCacheLockCache,
    ProfileEvents['FilesystemCacheReserveMicroseconds'] as FilesystemCacheReserve,
    ProfileEvents['FilesystemCacheEvictMicroseconds'] as FilesystemCacheEvict,
    ProfileEvents['FilesystemCacheGetOrSetMicroseconds'] as FilesystemCacheGetOrSet,
    ProfileEvents['FilesystemCacheGetMicroseconds'] as FilesystemCacheGet,
    ProfileEvents['FileSegmentWaitMicroseconds'] as FileSegmentWait,
    ProfileEvents['FileSegmentCompleteMicroseconds'] as FileSegmentComplete,
    ProfileEvents['FileSegmentLockMicroseconds'] as FileSegmentLock,
    ProfileEvents['FileSegmentWriteMicroseconds'] as FileSegmentWrite,
    ProfileEvents['FileSegmentUseMicroseconds'] as FileSegmentUse,
    ProfileEvents['FileSegmentRemoveMicroseconds'] as FileSegmentRemove,
    ProfileEvents['FileSegmentHolderCompleteMicroseconds'] as FileSegmentHolderComplete,
    ProfileEvents['WaitPrefetchTaskMicroseconds'] as WaitPrefetchTask,
    ProfileEvents['ThreadpoolReaderTaskMicroseconds'] as ThreadpoolReaderTask,
    ProfileEvents['ThreadpoolReaderPrepareMicroseconds'] as ThreadpoolReaderPrepare,
    ProfileEvents['ThreadpoolReaderSubmitReadSynchronouslyMicroseconds'] as ThreadpoolReaderSubmitReadSynchronously,
    ProfileEvents['ThreadpoolReaderSubmitLookupInCacheMicroseconds'] as ThreadpoolReaderSubmitLookupInCache,
    ProfileEvents['SleepFunctionElapsedMicroseconds'] as SleepFunctionElapsed,
    ProfileEvents['ThreadPoolReaderPageCacheHitElapsedMicroseconds'] as ThreadPoolReaderPageCacheHitElapsed,
    ProfileEvents['ThreadPoolReaderPageCacheMissElapsedMicroseconds'] as ThreadPoolReaderPageCacheMissElapsed,
    ProfileEvents['ConnectionPoolIsFullMicroseconds'] as ConnectionPoolIsFull,
    ProfileEvents['SharedMergeTreeVirtualPartsUpdateMicroseconds'] as SharedMergeTreeVirtualPartsUpdate,
    ProfileEvents['SharedMergeTreeVirtualPartsUpdatesFromZooKeeperMicroseconds'] as SharedMergeTreeVirtualPartsUpdatesFromZooKeeper,
    ProfileEvents['SharedMergeTreeVirtualPartsUpdatesFromPeerMicroseconds'] as SharedMergeTreeVirtualPartsUpdatesFromPeer,
    ProfileEvents['SharedMergeTreeMergeSelectingTaskMicroseconds'] as SharedMergeTreeMergeSelectingTask,
    ProfileEvents['SharedMergeTreeScheduleDataProcessingJobMicroseconds'] as SharedMergeTreeScheduleDataProcessingJob,
    ProfileEvents['SharedMergeTreeHandleFetchPartsMicroseconds'] as SharedMergeTreeHandleFetchParts,
    ProfileEvents['SharedMergeTreeHandleOutdatedPartsMicroseconds'] as SharedMergeTreeHandleOutdatedParts,
    ProfileEvents['SharedMergeTreeGetPartsBatchToLoadMicroseconds'] as SharedMergeTreeGetPartsBatchToLoad,
    ProfileEvents['SharedMergeTreeTryUpdateDiskMetadataCacheForPartMicroseconds'] as SharedMergeTreeTryUpdateDiskMetadataCacheForPart,
    ProfileEvents['SharedMergeTreeLoadChecksumAndIndexesMicroseconds'] as SharedMergeTreeLoadChecksumAndIndexes,
    ProfileEvents['StorageConnectionsElapsedMicroseconds'] as StorageConnectionsElapsed,
    ProfileEvents['DiskConnectionsElapsedMicroseconds'] as DiskConnectionsElapsed,
    ProfileEvents['HTTPConnectionsElapsedMicroseconds'] as HTTPConnectionsElapsed,
    ProfileEvents['ParquetFetchWaitTimeMicroseconds'] as ParquetFetchWaitTime,
    ProfileEvents['OSCPUWaitMicroseconds'] as OSCPUWait,
    ProfileEvents['OSIOWaitMicroseconds'] as OSIOWait,
    ProfileEvents['OSCPUVirtualTimeMicroseconds'] as OSCPUVirtualTimeMicroseconds
FROM system.query_log
WHERE event_time >= now() - INTERVAL last_hours_to_query HOUR
  AND type = 2
SETTINGS output_format_json_named_tuples_as_objects = 1, skip_unavailable_shards = 1</textarea>
                                <div class="form-check mt-3">
                                    <input class="form-check-input" type="checkbox" id="flushLogsCheckbox">
                                    <label class="form-check-label" for="flushLogsCheckbox">
                                        Run SYSTEM FLUSH LOGS
                                    </label>
                                </div>
                                <div class="form-check mt-2">
                                    <input class="form-check-input" type="checkbox" id="filterByDurationCheckbox" checked>
                                    <label class="form-check-label" for="filterByDurationCheckbox">
                                        Filter insignificant columns
                                    </label>
                                </div>
                                <div id="query-error" class="text-danger mt-2" style="display: none;"></div>
                            </form>
                        </div>
                    </div>
                </div>
                <div class="modal-footer">
                    <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Cancel</button>
                    <button type="button" class="btn btn-primary" id="btnDoLoad">Load</button>
                    <button type="button" class="btn btn-success" id="btnQueryClickhouse" style="display: none;">Query</button>
                    <div id="hourglass">⧗</div>
                </div>
            </div>
        </div>
    </div>

    <div id="legend-container"></div>
    <div id="table-container"></div>
    <div class="matrix-tooltip" style="visibility: hidden;"></div>
</body>
</html>

<script>

// Show correct button depending on tab
document.addEventListener('DOMContentLoaded', function () {
    const loadModal = document.getElementById('loadModal');
    loadModal.addEventListener('shown.bs.modal', function () {
        const tabButtons = document.querySelectorAll('#loadModalTabs button[data-bs-toggle="tab"]');
        tabButtons.forEach(button => {
            button.addEventListener('shown.bs.tab', function (e) {
                const target = e.target.getAttribute('data-bs-target');
                if (target === '#tabClickhouse') {
                    document.getElementById('btnDoLoad').style.display = 'none';
                    document.getElementById('btnQueryClickhouse').style.display = 'inline-block';
                } else {
                    document.getElementById('btnDoLoad').style.display = 'inline-block';
                    document.getElementById('btnQueryClickhouse').style.display = 'none';
                }
            });
        });
    });
});

// Handle ClickHouse query
let queryController = null;
document.getElementById('btnQueryClickhouse').addEventListener('click', async function () {
    if (queryController === null) {
        let rows = [];
        let error = '';
        const errorDiv = document.getElementById('query-error');
        errorDiv.style.display = 'none';
        errorDiv.textContent = '';
        document.getElementById('hourglass').style.display = 'inline-block';
        this.textContent = "Stop";

        if (document.getElementById('flushLogsCheckbox').checked) {
            queryController = new AbortController();
            await queryClickHouse({
                host: document.getElementById('url').value,
                user: document.getElementById('user').value,
                password: document.getElementById('password').value,
                query: "SYSTEM FLUSH LOGS",
                for_each_row: () => {},
                on_error: (errorMsg) => error = errorMsg,
                controller: queryController
            });
        }

        if (error === '') {
            queryController = new AbortController();
            await queryClickHouse({
                host: document.getElementById('url').value,
                user: document.getElementById('user').value,
                password: document.getElementById('password').value,
                query: document.getElementById('query').value,
                for_each_row: (data) => rows.push(data),
                on_error: (errorMsg) => error = errorMsg,
                controller: queryController
            });
        }

        queryController = null;
        this.textContent = "Query";
        document.getElementById('hourglass').style.display = 'none';

        if (error !== '') {
            errorDiv.textContent = error;
            errorDiv.style.display = 'block';
        } else {
            const filterByDuration = document.getElementById('filterByDurationCheckbox').checked;
            renderTable(handleData(parseClickHouseData(rows), filterByDuration));
            const loadModal = bootstrap.Modal.getInstance(document.getElementById('loadModal'));
            loadModal.hide();
        }
    } else { // Cancel query
        queryController.abort();
    }
});

// Handle uploaded JSON
document.getElementById('btnDoLoad').addEventListener('click', function () {
    const element = document.getElementById('loadFiles');
    const files = element.files;
    if (files.length <= 0) {
        return false;
    }

    const fr = new FileReader();
    fr.onload = function (e) {
        document.getElementById('query-error').style.display = 'none';
        const filterByDuration = document.getElementById('filterByDurationCheckbox').checked;
        renderTable(handleData(parseClickHouseData(JSON.parse(e.target.result).data), filterByDuration));
    };
    fr.readAsText(files.item(0));
    element.value = '';
    const loadModal = bootstrap.Modal.getInstance(document.getElementById('loadModal'));
    loadModal.hide();
});

function parseClickHouseData(rows) {
    let result = [];
    for (let i = 0; i < rows.length; i++) {
        let row = rows[i];
        for (let key in row) {
            if (key !== 'group' && !key.startsWith('_'))
                row[key] = +row[key];
        }
        result.push(row);
    }
    return result;
}

function renderTable(data) {
    // First calculate the available width for the table
    const containerWidth = document.getElementById("table-container").clientWidth;
    // Store it as a global variable so visualizations can access it
    window.availableWidth = containerWidth;

    // Calculate global maximum across all groups
    const globalMax = Math.max(...Object.values(data.groups).flatMap(group => group.max));
    // Store it for visualizations
    window.globalMaxValue = globalMax;

    d3.select("#table-container").selectAll("table").remove();

    const table = d3.select("#table-container")
        .append("table")
        .attr("class", "table sortable")
        .style("width", "100%"); // Ensure table takes full width

    const thead = table.append("thead");
    const tbody = table.append("tbody");

    // Get labels strting with underscore
    const extraColumns = data.labels.filter(label => label.startsWith("_")).map(label => label.substring(1));
    const columns = ["Group", "Count", ...extraColumns, "Covariance Matrix & Distribution"];
    thead.append("tr")
        .selectAll("th")
        .data(columns)
        .enter()
        .append("th")
        .text(d => d);

    const rows = tbody.selectAll("tr")
        .data(Object.entries(data.groups))
        .enter()
        .append("tr");

    // Put group name and count in the first two columns
    rows.append("td").text(([name]) => name);
    rows.append("td").text(([, stats]) => stats.count);

    // Iterate over extra columns and add them to the table
    extraColumns.forEach(col => {
        const index = data.labels.findIndex(label => label.substring(1) === col);
        rows.append("td").text(([, stats]) => stats.avg[index].toFixed(2));
    });

    // Add covariance matrix in the third column
    rows.append("td")
        .append("div")
        .attr("class", "matrix")
        .each(function ([, stats]) {
            addVisualization(this, data.labels, stats);
        });

    // Add sorting to the table
    sorttable.makeSortable(table.node());
}

function handleData(raw_data, filterByDuration = true) {
    // Process raw data
    const data = processData(raw_data);
    let globalAvg = computeGlobalAverages(data);

    // Filter out labels that have zero means across all groups
    const nonZeroMeanPredicate = (label, i, groups) => {
        // Check if all groups have zero mean for this label
        return !Object.values(groups).every(group => Math.abs(group.avg[i]) < 1e-10);
    };

    // Additional duration-based filtering
    const durationFilterPredicate = (label, i, groups) => {
        if (!filterByDuration || label.startsWith('_')) return true;

        const durationIndex = data.labels.indexOf('_duration');
        if (durationIndex === -1) return true; // If no _duration label, skip filtering

        // Check if any group has this metric > 1% of duration
        return Object.values(groups).some(group => {
            return group.p[i][90] > group.p[durationIndex][50] * 0.05;
        });
    };

    // Apply both filters
    const combinedPredicate = (label, i, groups) => {
        return nonZeroMeanPredicate(label, i, groups) && durationFilterPredicate(label, i, groups);
    };

    const filteredData = filterData(data, combinedPredicate);
    globalAvg = computeGlobalAverages(filteredData); // Recompute global averages after filtering

    // Split labels into underscore-prefixed and regular labels while preserving original order
    const underscoreLabels = filteredData.labels.filter(label => label.startsWith('_'));
    const regularLabels = filteredData.labels.filter(label => !label.startsWith('_'));

    // Create comparison function to sort regular labels by global averages
    const compByAvg = (a, b) => {
        const aIdx = filteredData.labels.indexOf(a);
        const bIdx = filteredData.labels.indexOf(b);
        return globalAvg[bIdx] - globalAvg[aIdx]; // Sort descending
    };

    // Sort only regular labels, keeping underscore labels in original order
    const sortedRegularLabels = regularLabels.sort(compByAvg);

    // Merge back in the correct order: underscore labels followed by sorted regular labels
    const orderedLabels = [...underscoreLabels, ...sortedRegularLabels];

    // Order data using the merged labels
    return orderLabels(filteredData, (a, b) => {
        return orderedLabels.indexOf(a) - orderedLabels.indexOf(b);
    });
}

drawLegend("#legend-container");

// Render example data
renderTable(handleData(generateTestData()));

</script>

<script language="javascript" type="text/javascript" src="test_env.js"></script>
